package com.hadooop.dedupe;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class DedupeDriver {
    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {

        Configuration conf = new Configuration();
        conf.set("mapreduce.framework.name", "local");
        //conf.set("fs.defaultFS", "hdfs://hadoop0:9000/");
        //System.setProperty("HADOOP_USER_NAME", "root");

        Job job = Job.getInstance();

        job.setMapperClass(DedupeMapper.class);
        job.setReducerClass(DedupeReducer.class);
        job.setJarByClass(DedupeDriver.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

//        FileInputFormat.setInputPaths(job, new Path("hdfs://hadoop0:9000/deduplication/input"));
//        FileOutputFormat.setOutputPath(job, new Path("hdfs://hadoop0:9000/deduplication/output"));
        FileInputFormat.setInputPaths(job, new Path("hadoop-02-mapreduce/file/deduplication/input"));
        FileOutputFormat.setOutputPath(job, new Path("hadoop-02-mapreduce/file/deduplication/output"));

        System.exit(job.waitForCompletion(true)? 0 : -1);

    }
}
